import Core.MongoDB as MongoDB
import Core.Gadget as Gadget
import Core.IO as IO
import datetime
import pandas as pd
import numpy as np
from Core.Config import *


def SymbolWithIndustry(database, industryField="CSIndustry1"):
    #
    instruments = database.find("Instruments", "Stock")
    mapping = IndustryMapping()
    #
    data = []
    for instrument in instruments:
        #
        symbol = instrument["Symbol"]
        industry = instrument[industryField]

        if industry not in mapping.keys():
            print(symbol, "No Industry", industry)
            continue
        else:
            industry = mapping[industry]
        #
        data.append(symbol, industry)
    #
    df = pd.DataFrame(data, columns=["Symbol", "Industry"])
    return df

def PrintIndustry(database, industryField="CSIndustry1"):
    instrumentListByIndustry = IndustryHirachy(database, industryField)
    for key, value in instrumentListByIndustry.items():
        print(key, len(value))


#--- BuildIndustries GetIndustry---
# CSIndustry1
def IndustryHirachy(database, industryField="CSIndustry1"):

    instruments = database.find("Instruments", "Stock")

    #
    mapping = IndustryMapping()

    #
    industryType = "CS" # CS

    #构建行业的列表（有多少行业）
    industries = []
    instrumentListByIndustry = {}
    for instrument in instruments:
        #
        symbol = instrument["Symbol"]
        datetime1 = instrument["DateTime1"]
        datetime2 = instrument["DateTime2"]

        #
        if industryField not in instrument:
            print(instrument["Symbol"] + " No Industry info")
            continue
        industry = instrument[industryField]

        #
        if industry == None:
            print(symbol, "None Industry", industry,
                  instrument["Industry"], instrument["SWIndustry1"],
                  datetime1, datetime2)
            continue

        if industry not in mapping.keys():
            print(symbol, "No Industry", industry, datetime1, datetime2)
            continue
        else:
            industry = mapping[industry]

        # 这一步就把所有行业都放进去了 进了一个list ，下面循环这个就行了
        if industry not in industries:
            industries.append(industry)
            instrumentListByIndustry[industry] = []
        #
        instrumentListByIndustry[industry].append(instrument["Symbol"])
    return instrumentListByIndustry

# ---
def CalcIndustryFactor(instruments):
    keywords = {}
    start = True
    for instrument in instruments:
        symbol = instrument["Symbol"]
        if symbol == "000003.SZ":
            start = True
        if start == False:
            continue

        print(symbol)
        if instrument["Properties"].get("SWIndustry1") == None:
            continue
        industry1 = "1" + instrument["Properties"]["SWIndustry1"]
        industry2 = "2" + instrument["Properties"]["SWIndustry2"]
        industry3 = "3" + instrument["Properties"]["SWIndustry3"]

        statements = database.findWithFilter("Fundamental",symbol + "_Fundamental")
        for statement in statements:
            if "Values" in statement and "SegmentSalesByProduct" in statement["Values"]:
                if statement["Period"] == 1 or statement["Period"] == 3:
                    continue
                salesSegment = statement["Values"]["SegmentSalesByProduct"]
                print(salesSegment)
                if salesSegment == 0:
                    continue
                segments = salesSegment.split(";")
                countSegments = len(segments)
                for i in range(countSegments):
                    if i >= 3:
                        continue # 只关注前三项业务
                    segment = segments[i]
                    segment2 = segment.split(":")
                    #---
                    title = None
                    if len(segment2) <= 2:
                        title = segment2[0]
                    else:
                        title = segment2[0] + ":" + segment2[1]
                    #---
                    if title != None and title not in keywords:
                        keywords[title] = {}
                    #---和三级行业的Connection---
                    if keywords[title].get(industry1) == None:
                        keywords[title][industry1] = 0
                    if keywords[title].get(industry2) == None:
                        keywords[title][industry2] = 0
                    if keywords[title].get(industry3) == None:
                        keywords[title][industry3] = 0

                    keywords[title][industry1] = keywords[title][industry1] + 1
                    keywords[title][industry2] = keywords[title][industry2] + 1
                    keywords[title][industry3] = keywords[title][industry3] + 1
                kkwoodc = 0
        #---Loop Statment---
        kkwood = 2
    kkwood = 3
    for keyword,industryByFreq in keywords.items():
        s = ""
        freq1 = 0
        freq2 = 0
        freq3 = 0
        industry1 = None
        industry2 = None
        industry3 = None
        for industry,freq in industryByFreq.items():
            s = s + industry + "," + str(freq) + ","
            if industry[0] == "1" and freq > freq1:
                industry1 = industry[1:]
            if industry[0] == "2" and freq > freq2:
                industry2 = industry[1:]
            if industry[0] == "3" and freq > freq3:
                industry3 = industry[1:]

        line = keyword + "," + s
        line2 = keyword + "," + industry1 + "," + industry2 + "," + industry3
        print(line2)
        IO.WriteToFile("d:/ProductKeyword.csv",line)
        IO.WriteToFile("d:/ProductKeywordClean.csv",line2)


# ---Industry Alpha Analysis---
def CalcIndustryAlpha(database, datetime1, datetime2):

    industryInstruments = database.findWithFilter("Instruments", "Index", filter={"Description":{"$regex":"(申万)"}})
    returnByIndustry = {}
    bmSymbol = "000001.SH"
    returnByIndustry[bmSymbol] = []

    #---Dscription---
    decsBySymbol = {}
    for instrument in industryInstruments:
        decsBySymbol[instrument["Symbol"]] = instrument["Description"]

    datetimes = Gadget.GenerateMonthDates(datetime1, datetime2)

    for dt in datetimes:
        #
        datetime2 = Gadget.ToUTCDateTime(dt)
        bmQuotes = database.getDataSeries("000001.SH_Time_86400_Bar", "Index", datetime1, datetime2)
        length = bmQuotes.Count()
        if length == 0:
            continue
        first = bmQuotes.First()
        last = bmQuotes.Last()
        print(bmQuotes.First()["DateTime"] + " to " + bmQuotes.Last()["DateTime"])
        bmReturn = last["Close"] / first["Close"] - 1
        #
        returnByIndustry[bmSymbol].append([last["DateTime"], bmReturn])

        #
        for instrument in industryInstruments:
            symbol = instrument["Symbol"]
            quotes = database.getDataSeries(symbol + "_Time_86400_Bar", "Index", datetime1, datetime2)
            first = quotes.First()
            last = quotes.Last()
            length = quotes.Count()

            # ---add to Cache---
            if length > 0:
                ret = last["Close"] / first["Close"] - 1
                alpha = ret - bmReturn

                if symbol not in returnByIndustry:
                    returnByIndustry[symbol] = []
                #
                returnByIndustry[symbol].append([last["DateTime"], ret, alpha])
            kkwood = 0

        datetime2 = last["StdDateTime"]
        datetime1 = datetime2
    # ---Loop DataTimes--

    # --- Print1 ----
    count = len(returnByIndustry[bmSymbol])

    # ---Return Table---
    for iMonth in range(count):
        header = "Date,BM"
        strLine = returnByIndustry[bmSymbol][iMonth][0] + "," + str(returnByIndustry[bmSymbol][iMonth][1])
        for symbol, dataList in returnByIndustry.items():
            if symbol == bmSymbol:
                continue
            dataSeries = returnByIndustry[symbol]
            header += "," + decsBySymbol[symbol]
            strLine += "," + str(dataSeries[iMonth][1])
        if iMonth == 0:
            print(header)
        print(strLine)

    kkwood = 1

    # ---Print Alpha Table---
    for iMonth in range(count):
        header = "Date,BM"
        strLine = returnByIndustry[bmSymbol][iMonth][0] + "," + str(returnByIndustry[bmSymbol][iMonth][1])
        for symbol, dataList in returnByIndustry.items():
            if symbol == bmSymbol:
                continue
            dataSeries = returnByIndustry[symbol]
            header += "," + decsBySymbol[symbol]
            strLine += "," + str(dataSeries[iMonth][2])
        if iMonth == 0:
            print(header)
        print(strLine)

    kkwood = 2

    # --- Print2 ---
    for symbol, dataList in returnByIndustry.items():
        print(symbol + " ")
        for item in dataList:
            print(item[0] + "," + item[1] + "," + item[2])
        kkwood = 1

    kkwood = 0


def IndustryMappedByPeriod():
    superClass = {}
    superClass["Periodic"] = []
    superClass["Consumption"] = []
    superClass["Tech"] = []
    superClass["Finance"] = []
    #
    superClass["Periodic"].append("煤炭")
    superClass["Periodic"].append("石油石化")
    superClass["Periodic"].append("化工")
    superClass["Periodic"].append("基础化工")
    superClass["Periodic"].append("有色金属")
    superClass["Periodic"].append("钢铁")
    superClass["Periodic"].append("建筑")
    superClass["Periodic"].append("建材")
    superClass["Periodic"].append("机械")
    superClass["Periodic"].append("电力设备")
    superClass["Periodic"].append("交通运输")
    #
    superClass["Consumption"].append("农林牧渔")
    superClass["Consumption"].append("轻工制造")
    superClass["Consumption"].append("家电")
    superClass["Consumption"].append("汽车") #189
    superClass["Consumption"].append("医药")
    superClass["Consumption"].append("食品饮料")
    superClass["Consumption"].append("纺织服装")
    superClass["Consumption"].append("商贸零售")
    superClass["Consumption"].append("餐饮旅游")
    #
    superClass["Tech"].append("国防军工")
    superClass["Tech"].append("新能源") # 能源，汽车
    superClass["Tech"].append("传媒")
    superClass["Tech"].append("计算机")
    superClass["Tech"].append("通信")
    superClass["Tech"].append("电子元器件")
    #
    superClass["Finance"].append("房地产")
    superClass["Finance"].append("银行")
    superClass["Finance"].append("非银行金融") #49
    #
    return superClass


def IndustryMapping():
    #
    mapping = {}

    industries = []

    # ---周期 上游---
    industries.append("煤炭")
    industries.append("石油石化")
    #
    mapping["采掘"] = "钢铁" #3, 600762.SH

    # ---周期 中游---
    # industries.append("化工") #47
    industries.append("基础化工") #265
    industries.append("有色金属")
    industries.append("钢铁")
    industries.append("建筑")
    industries.append("建材")
    industries.append("机械")
    industries.append("电力设备")
    industries.append("交通运输") # 119
    #
    mapping["化工"] = "基础化工" # 3
    #
    mapping["建筑材料"] = "建材" # 3
    mapping["建筑建材"] = "建筑" # 1 600286
    mapping["建筑装饰"] = "建筑" # 16 具体拆分
    #
    mapping["机械设备"] = "机械"
    mapping["电气设备"] = "电力设备"

    # ---消费 下游---
    industries.append("农林牧渔")
    industries.append("轻工制造")
    industries.append("家电")
    industries.append("汽车") #189
    industries.append("医药")
    industries.append("食品饮料")
    industries.append("纺织服装")
    industries.append("商贸零售")
    industries.append("餐饮旅游")
    #
    mapping["家用电器"] = "家电"
    mapping["医药生物"] = "医药"
    mapping["商业贸易"] = "商贸零售" #6

    # ---科技---
    industries.append("国防军工")
    industries.append("新能源") # 能源，汽车
    industries.append("传媒")
    industries.append("计算机")
    industries.append("通信")
    industries.append("电子元器件")
    #
    mapping["电子"] = "电子元器件" #25
    mapping["信息服务"] = "计算机" #1， 000583.SZ

    # ---金融---
    industries.append("房地产")
    industries.append("银行")
    # industries.append("券商")
    industries.append("非银行金融") #49
    #
    mapping["非银金融"] = "非银行金融" #6

    # ---其他---
    industries.append("公用事业") #7
    industries.append("综合")

    mapping["电力及公用事业"] = "公用事业" #147

    for name in industries:
        mapping[name] = name

    return mapping


def UpdateIndustryHirachy(datetime1, datetime2):

    instruments = database.find("Instruments", "Stock")
    #
    industryType = "CS"  # CS

    # 构建行业的列表（有多少行业）

    mapping = IndustryMapping()

    instrumentListByIndustry = {}
    industryBySymbol = {}

    for instrument in instruments:
        symbol = instrument["Symbol"]
        # print(symbol)
        if industryType + "Industry1" not in instrument:
            print(instrument["Symbol"] + " No Industry info")
            continue

        industry = instrument[industryType + "Industry1"]
        datetime1 = instrument["DateTime1"]
        datetime2 = instrument["DateTime2"]

        #
        if industry == None:
            print(symbol, "None Industry", industry, instrument["Industry"], instrument["SWIndustry1"],  datetime1, datetime2)
            continue

        if industry not in mapping.keys():
            print(symbol, "No Industry", industry, datetime1, datetime2)
            continue
        else:
            industry = mapping[industry]

        database.Update("Instruments", "Stock", {"Symbol": symbol}, {"Industry": industry})

    pass


def CheckIndustryVariation2():
    #
    dataSeries = database.Find("Factor", "CSIndustry")
    mapping = IndustryMapping()

    for d in dataSeries:
        industry = d["Value"]
        symbol = d["Symbol"]
        dt = d["StdDateTime"]
        if industry not in mapping.keys():
            print(symbol, "No Industry", industry, dt)


def CheckIndustryVariation():

    instruments = database.find("Instruments", "Stock")
    mapping = IndustryMapping()

    count = 0
    for instrument in instruments:
        symbol = instrument["Symbol"]
        dataSeries = database.Find("Factor", "CSIndustry", {"Symbol": symbol})

        lastIndustry = None
        changeDate = []
        for d in dataSeries:
            industry = d["Value"]
            dt = d["StdDateTime"]
            if lastIndustry != None and industry != lastIndustry:
                changeDate.append([industry, dt.date()])
            lastIndustry = industry

        if len(changeDate) > 1:
            print(symbol, changeDate)
            count += 1

        k = 0
    print("Total", count)


def Automatic_BuildEquallyWeightIndex(database, datetime1, datetime2, industry=None):
    # ---load Data---
    IO.LoadMultiInstrumentsBarsAsDataFrame()


def BuildEquallyWeightIndex(database, datetime1, datetime2, industryList=["Total"], saveToDateBase=False):

    datetime1 = Gadget.ToUTCDateTime(datetime1)
    datetime2 = Gadget.ToUTCDateTime(datetime2)

    # ---Prepare Industry---
    mapping = IndustryMapping()

    # ---Load Benchmark---
    filter = {}
    filter["StdDateTime"] = {}
    filter["StdDateTime"]["$gte"] = datetime1
    filter["StdDateTime"]["$lte"] = datetime2
    filter["Symbol"] = "000001.SH"
    bmBarSeries = database.Find("Index", "DailyBar", filter)

    # ---Prepare Cache---
    indicator = {}
    data = {}
    missingSymbol = []
    for industry in industryList:
        indicator[industry] = 1
        data[industry] = []

    # ---Loop Days---
    for bar in bmBarSeries:
        stdDatetime = bar["StdDateTime"]
        print(stdDatetime)

        # ---Exclude New Stock---
        earlyDate = stdDatetime + datetime.timedelta(days=-365)
        listedInstrument = Gadget.FindListedInstrument(database, stdDatetime)
        symbols = {}
        symbolToIndustry = {}

        for industry in industryList:
            symbols[industry] = []

        # ---按照Industry对Symbol分组---
        for instrument in listedInstrument:
            symbol = instrument["Symbol"]

            # ---Exclude New Stock---
            if instrument["DateTime1"] > earlyDate:
                continue

            # ---Exclude Non-Target Industry---
            originalIndustry = instrument["Industry"]


            thisIndustry = None
            if originalIndustry not in mapping.keys():
                # print(symbol, "Missing Industry", originalIndustry)
                # continue
                pass
            else:
                thisIndustry = mapping[originalIndustry]

            #
            if "Total" in industryList:
                thisIndustry = "Total"

            #
            if thisIndustry not in symbols.keys():
                print(symbol, thisIndustry, "Missing Industry")
                continue

            #
            symbols[thisIndustry].append(symbol)
            symbolToIndustry[symbol] = thisIndustry


        # ---Find Total "Return Values" at a Specific Day---
        validReturn = {}
        for industry in industryList:
            validReturn[industry] = []

        # gte = {"Value": {"$gte": -0.11}}
        # lte = {"Value": {"$lte": 0.11}}
        # filter = {"$and": [gte, lte]}
        filter = {}
        filter["StdDateTime"] = stdDatetime
        #
        returns = database.Find("Factor", "DailyReturn", filter)
        for r in returns:
            symbol = r["Symbol"]
            if symbol not in symbolToIndustry.keys():
                # print(symbol, "Not Valid Stock")
                continue

            thisIndustry = symbolToIndustry[symbol]

            # if find, pop up
            symbols[thisIndustry].remove(symbol)

            # 排除可能的错误
            if r["Value"] > 0.11 or r["Value"] < -0.11:
                continue
            #
            validReturn[thisIndustry].append(r["Value"])

        # ---Check Missing---
        for industry in industryList:
            tempList = symbols[industry]
            if len(tempList) > 0:
                print("Missing Ret", industry, stdDatetime, tempList)
                for symbol in tempList:
                    missingSymbol.append([industry, stdDatetime.date(), symbol])

        # ---Check Abnormal value---
        # if stdDatetime.month == 2 and stdDatetime.day == 12:
        #    PrintSpecificDayReturn(stdDatetime, returns ,listedSymbols)

        # ---Calculating---
        for industry in industryList:
            mean = np.mean(validReturn[industry])
            if not np.isnan(mean):
                indicator[industry] *= 1 + mean

            # std = np.std(validReturn)
            result = [stdDatetime.date(), mean, indicator[industry], len(validReturn[industry]), len(symbols[industry])]
            print(industry, result)
            data[industry].append(result)

        pass

    #
    for industry in industryList:
        dfIndex = pd.DataFrame(data[industry], columns=["DateTime", "Mean", "Index", "Count", "Missing"])
        dfIndex.to_csv("D:/Data/IndustryAnalysis/IndustryIndex/" + industry + "_EquallyWeightIndex.csv")

    #
    dfMissingSymbols = pd.DataFrame(missingSymbol, columns=["Industry", "DateTime", "Symbol"])
    dfMissingSymbols.to_csv("D:/Data/IndustryAnalysis/IndustryIndex/Missing_Symbols_EquallyWeightIndex.csv")


if __name__ == '__main__':
    # ---Connect to DataBase, Find Series 连接数据库---
    from Core.Config import *
    cfgPathFilename = os.getcwd() + "/../config.json"
    config = Config(cfgPathFilename)
    database = config.DataBase("Mongo")
    realtime = config.RealTime()

    dt = datetime.datetime(2018,11,27,15,0,0)
    dt = Gadget.ToUTCDateTime(dt)

    datetime1 = datetime.datetime(2005, 1, 1)
    datetime2 = datetime.datetime(2019, 5, 1)

    #
    PrintIndustry(database)

    #
    instrumentList = database.Find("Instruments", "Stock", {"SWIndustry1": "家用电器"})
    for instrument in instrumentList:
        print(instrument["Symbol"], instrument["DateTime1"].date(), instrument["DateTime2"].date())

    # BuildEquallyWeightIndex(database, datetime1, datetime2)
    #
    # #
    # mapping = IndustryMapping()
    # industryList = []
    # for key, value in mapping.items():
    #     if value not in industryList:
    #         industryList.append(value)

    # BuildEquallyWeightIndex(database, datetime1, datetime2, industryList)

    #
    datetime1 = datetime.datetime(2005,1,1)
    datetime2 = datetime.datetime(2019,5,1)
    # UpdateIndustryHirachy(datetime1, datetime2)

    # CheckIndustryVariation()
    # CheckIndustryVariation2()
    #
    instrumentListByIndustry = PrintIndustry(database)
    s = instrumentListByIndustry["国防军工"]
    s1 = instrumentListByIndustry["医药"]

    s2 = instrumentListByIndustry["商业贸易"]
    s3 = instrumentListByIndustry["商贸零售"]

    s4 = instrumentListByIndustry["电子"]
    s5 = instrumentListByIndustry["电子元器件"]

    s4 = instrumentListByIndustry["机械"]
    s5 = instrumentListByIndustry["机械设备"]

    s6 = instrumentListByIndustry["公用事业"]
    s7 = instrumentListByIndustry["电力及公用事业"]

    s8 = instrumentListByIndustry["煤炭"]
    s9 = instrumentListByIndustry["石油石化"]
    s10 = instrumentListByIndustry["采掘"]

    s11 = instrumentListByIndustry["信息服务"]

    s12 = instrumentListByIndustry["建筑材料"]
    s13 = instrumentListByIndustry["建筑建材"]

    # ---证券列表---
    filter = {}
    #filter = {"Symbol":"000001.SZ"}
    filter["limit"] = 100
    # instruments = database.findWithFilter("Instruments","Stock",filter)

    # PrintIndustry(database)
    # CalcIndustryFactor(instruments)

    datetime1 = datetime.datetime(2000, 1, 1)
    datetime2 = datetime.datetime(2018, 5, 14)
    datetime1 = Gadget.ToUTCDateTime(datetime1)
    datetime2 = Gadget.ToUTCDateTime(datetime2)
    # CalcIndustryAlpha(database, datetime1, datetime2)

